{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [
    {
     "data": {
      "text/plain": "          UserID    微博数\n0     1330417035    655\n1     1679622967    432\n2     1692504434  20914\n3     1746598137   1280\n4     2270723560  12802\n...          ...    ...\n1479  6353077287    418\n1480  7803057621    130\n1481  5997225853   5631\n1482  3940276371    198\n1483  5172202845  13829\n\n[1484 rows x 2 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>UserID</th>\n      <th>微博数</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1330417035</td>\n      <td>655</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1679622967</td>\n      <td>432</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1692504434</td>\n      <td>20914</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1746598137</td>\n      <td>1280</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>2270723560</td>\n      <td>12802</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1479</th>\n      <td>6353077287</td>\n      <td>418</td>\n    </tr>\n    <tr>\n      <th>1480</th>\n      <td>7803057621</td>\n      <td>130</td>\n    </tr>\n    <tr>\n      <th>1481</th>\n      <td>5997225853</td>\n      <td>5631</td>\n    </tr>\n    <tr>\n      <th>1482</th>\n      <td>3940276371</td>\n      <td>198</td>\n    </tr>\n    <tr>\n      <th>1483</th>\n      <td>5172202845</td>\n      <td>13829</td>\n    </tr>\n  </tbody>\n</table>\n<p>1484 rows × 2 columns</p>\n</div>"
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tbasedata = pd.read_csv('../data/basedata.csv')\n",
    "basedata = tbasedata.head(1)\n",
    "tbasedata[['UserID', '微博数']]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "def getReqFromUid(uid, since_id):\n",
    "    if since_id == \"\":\n",
    "        url = \"https://m.weibo.cn/api/container/getIndex?type=uid&value=\" + uid + \"&containerid=107603\" + uid\n",
    "    else:\n",
    "        url = \"https://m.weibo.cn/api/container/getIndex?type=uid&value=\" + uid + \"&containerid=107603\" + uid + \"&since_id=\" + since_id\n",
    "    headers = {\n",
    "        'Host': 'm.weibo.cn',\n",
    "        'X-Requested-With': 'XMLHttpRequest',\n",
    "        'Sec-Fetch-Site': 'same-origin',\n",
    "        'X-XSRF-TOKEN': '73490b',\n",
    "        'Accept-Language': 'zh-CN,zh-Hans;q=0.9',\n",
    "        'Accept-Encoding': 'gzip, deflate, br',\n",
    "        'Sec-Fetch-Mode': 'cors',\n",
    "        'Accept': 'application/json, text/plain, */*',\n",
    "        'MWeibo-Pwa': '1',\n",
    "        'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.44(0x18002c2b) NetType/WIFI Language/zh_CN',\n",
    "        'Connection': 'keep-alive',\n",
    "        'Sec-Fetch-Dest': 'empty',\n",
    "        'Referer': 'https://m.weibo.cn/u/' + uid\n",
    "    }\n",
    "    cookies = {\n",
    "        'XSRF-TOKEN': '73490b',\n",
    "        'MLOGIN': '0',\n",
    "        'M_WEIBOCN_PARAMS': 'fid%3D1005056295222109%26uicode%3D10000011',\n",
    "        'WEIBOCN_FROM': '1110006030',\n",
    "        '_T_WM': '25879493818',\n",
    "        'mweibo_short_token': '126413f11c'\n",
    "\n",
    "    }\n",
    "    response = requests.get(url, headers=headers, cookies=cookies)\n",
    "    return response"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "outputs": [],
   "source": [
    "def getText(uid, since_id,textList):\n",
    "    resp = getReqFromUid(uid, since_id)\n",
    "    weibo_data_text = json.loads(resp.text)\n",
    "    # print(weibo_data_text)\n",
    "    if weibo_data_text.get('data'):\n",
    "        since_id = weibo_data_text['data']['cardlistInfo']['since_id']\n",
    "        # print(since_id)\n",
    "        for textdata in weibo_data_text['data']['cards']:\n",
    "            textList.append(textdata['mblog']['text'].split(\">//<\")[0])\n",
    "    res = {'since_id':str(since_id),'textList':textList}\n",
    "    return res\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "def get_WB_text_data(tbasedata,mode):\n",
    "\n",
    "    start_time = time.time()  # 获取当前时间戳\n",
    "    if mode==1:\n",
    "        idList = tbasedata[['UserID']]\n",
    "        print(idList.head(1))\n",
    "    else:\n",
    "        idList = tbasedata[['UserID', '微博数']]\n",
    "    bddPD = pd.DataFrame()\n",
    "    wbcount=0\n",
    "    for i in range(len(idList)):\n",
    "    # for i in range(2,3):\n",
    "        if i%10==0:\n",
    "            print(i)\n",
    "            end_time = time.time()  # 获取当前时间戳\n",
    "            elapsed_time = end_time - start_time  # 计算时间差\n",
    "            print(\"任务执行时间：\", elapsed_time, \"秒\")\n",
    "\n",
    "        uid = str(idList.iloc[i]['UserID'])\n",
    "        if mode==1:\n",
    "            wbcount==100\n",
    "        else:\n",
    "            wbcount = idList.iloc[i]['微博数']\n",
    "        textList = []\n",
    "        #发起请求\n",
    "        try:\n",
    "            since_id = ''\n",
    "            #最多获取100条\n",
    "            pagecount = wbcount//10\n",
    "            if  pagecount > 10:\n",
    "                pagecount = 10\n",
    "            for j in range(pagecount):\n",
    "                res = getText(uid, since_id,textList)\n",
    "                since_id = res['since_id']\n",
    "                # print(since_id)\n",
    "                textList = res['textList']\n",
    "                time.sleep(0.3)\n",
    "            testpd = pd.DataFrame({'UserID': [uid], 'text': [textList]})\n",
    "            bddPD = bddPD.append(testpd, ignore_index=True)\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            testpd = pd.DataFrame({'UserID': [uid], 'text': [[]]})\n",
    "            bddPD = bddPD.append(testpd, ignore_index=True)\n",
    "    return bddPD\n",
    "\n",
    "\n",
    "# basedata  = pd.merge(basedata,bddPD,on=['UserID'])\n",
    "# basedata.head(10)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "wb_text_data = pd.read_csv('../data/WB_text.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "outputs": [
    {
     "data": {
      "text/plain": "        Unnamed: 0        UserID\ncount  1484.000000  1.484000e+03\nmean    741.500000  5.943431e+09\nstd     428.538213  1.765528e+09\nmin       0.000000  1.000562e+09\n25%     370.750000  5.391134e+09\n50%     741.500000  6.390966e+09\n75%    1112.250000  7.383720e+09\nmax    1483.000000  7.881224e+09",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Unnamed: 0</th>\n      <th>UserID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>1484.000000</td>\n      <td>1.484000e+03</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>741.500000</td>\n      <td>5.943431e+09</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>428.538213</td>\n      <td>1.765528e+09</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>0.000000</td>\n      <td>1.000562e+09</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>370.750000</td>\n      <td>5.391134e+09</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>741.500000</td>\n      <td>6.390966e+09</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>1112.250000</td>\n      <td>7.383720e+09</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>1483.000000</td>\n      <td>7.881224e+09</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wb_text_data.describe()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "287"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "erData = wb_text_data[wb_text_data['text']=='[]']\n",
    "len(erData)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       UserID\n",
      "3  1746598137\n",
      "0\n",
      "任务执行时间： 0.00397038459777832 秒\n",
      "10\n",
      "任务执行时间： 0.01801776885986328 秒\n",
      "20\n",
      "任务执行时间： 0.0298311710357666 秒\n",
      "30\n",
      "任务执行时间： 0.0408022403717041 秒\n",
      "40\n",
      "任务执行时间： 0.05077815055847168 秒\n",
      "50\n",
      "任务执行时间： 0.057755470275878906 秒\n",
      "60\n",
      "任务执行时间： 0.0687263011932373 秒\n",
      "70\n",
      "任务执行时间： 0.0796973705291748 秒\n",
      "80\n",
      "任务执行时间： 0.08867335319519043 秒\n",
      "90\n",
      "任务执行时间： 0.09665203094482422 秒\n",
      "100\n",
      "任务执行时间： 0.10363578796386719 秒\n",
      "110\n",
      "任务执行时间： 0.11165308952331543 秒\n",
      "120\n",
      "任务执行时间： 0.11958885192871094 秒\n",
      "130\n",
      "任务执行时间： 0.12756943702697754 秒\n",
      "140\n",
      "任务执行时间： 0.1375415325164795 秒\n",
      "150\n",
      "任务执行时间： 0.14808368682861328 秒\n",
      "160\n",
      "任务执行时间： 0.16005301475524902 秒\n",
      "170\n",
      "任务执行时间： 0.17002654075622559 秒\n",
      "180\n",
      "任务执行时间： 0.17800426483154297 秒\n",
      "190\n",
      "任务执行时间： 0.18598318099975586 秒\n",
      "200\n",
      "任务执行时间： 0.1959836483001709 秒\n",
      "210\n",
      "任务执行时间： 0.20293736457824707 秒\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\25008\\AppData\\Local\\Temp\\ipykernel_4508\\763388958.py:39: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  bddPD = bddPD.append(testpd, ignore_index=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "220\n",
      "任务执行时间： 0.2119143009185791 秒\n",
      "230\n",
      "任务执行时间： 0.2197415828704834 秒\n",
      "240\n",
      "任务执行时间： 0.22691130638122559 秒\n",
      "250\n",
      "任务执行时间： 0.2360067367553711 秒\n",
      "260\n",
      "任务执行时间： 0.24682092666625977 秒\n",
      "270\n",
      "任务执行时间： 0.25479912757873535 秒\n",
      "280\n",
      "任务执行时间： 0.26377439498901367 秒\n"
     ]
    }
   ],
   "source": [
    "bddPD = get_WB_text_data(erData,1)\n",
    "\n",
    "# bddPD.to_csv('../data/WB_text2.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ISFJ']\n"
     ]
    }
   ],
   "source": [
    "print(tbasedata[tbasedata['UserID']==1746598137]['MBTI类型'].values)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1484\n",
      "1197\n"
     ]
    }
   ],
   "source": [
    "allWBUserData = pd.read_csv('../data/mbti_weibo_data.csv')\n",
    "allWBTextData = pd.read_csv('../data/WB_text.csv')\n",
    "allWBUserData = pd.merge(allWBUserData,allWBTextData,on='UserID',how='left')\n",
    "print(len(allWBUserData))\n",
    "allWBUserData = allWBUserData.drop(allWBUserData[allWBUserData['text']=='[]'].index)\n",
    "print(len(allWBUserData))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "outputs": [],
   "source": [
    "allWBUserData.to_csv('../data/mbti_weibo_data_all.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}